#!/usr/bin/python -O

"""
    Utility module for processing the active user dataformat.
"""

import collections
import itertools
import os
import sys
import datetime

import wp_datetime as dt
import wp_namespace as ns


START_COHORT = 2003
END_COHORT = 2012
METRICS = [
    'edits',
    'bytes',
    'hours',
]

END_YEAR = 2012
CURRENT_CUTOFF = '2011-05-03'
CAP_YEAR = True


def warn(message):
    sys.stderr.write(timestamp() + ' ' + message + '\n')

def timestamp():
    return datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')

class EditRecord(object):
    __slots__ = [
        'username',
        'timestamp',
        'offset',
        'lifespan',
        'cohort',
        'namespace',
        'revert_status',
        'duration',
        'bytes_added',
        'bytes_removed',
        'bytes_changed',
    ]

    def __init__(self, record_dict):
        self.username = record_dict['username']
        self.timestamp = record_dict['timestamp']
        self.offset = record_dict['offset']
        self.lifespan = record_dict['lifespan']
        self.cohort = record_dict['cohort']
        self.namespace = intern(record_dict['namespace'])
        self.revert_status = intern(record_dict['revert_status'])
        self.duration = record_dict['duration']
        self.bytes_added = record_dict['bytes_added']
        self.bytes_removed = record_dict['bytes_removed']
        self.bytes_changed = record_dict['bytes_changed']

    def get_work_type(self):
        if self.revert_status == 'revert':
            return 'maintenance'
        elif self.namespace == 'main':
            return 'article'
        elif self.namespace == 'talk':
            return 'article talk'
        elif self.namespace == 'user':
            return 'user'
        elif self.namespace == 'user talk':
            return 'user talk'
        else:
            return 'other'

    def get_year(self):
        return str(self.cohort + self.offset)

    def __getitem__(self, key):
        key = key.replace(' ', '')
        if key in self.__slots__:
            return getattr(self, key)
        elif key == 'worktype':
            return self.get_work_type()
        elif key == 'year':
            return self.get_year()
        else:
            raise KeyError(key)

    def get_metric(self, key):
        if key == 'hours':
            return 1.0 * self.duration / 3600.0
        elif key == 'bytes':
            return abs(self.bytes_changed)
        elif key == 'edits':
            return 1

class UserIterator:
    """Iterates through a set of edit data files in a directory."""
    def __init__(self, input_dir):
        self.paths = [os.path.join(input_dir, p) for p in os.listdir(input_dir)]
        self.paths = [p for p in self.paths if os.path.isfile(p)]
        warn('reading %s files in %s' % (len(self.paths), `input_dir`))
        self.handle = None
        self.buffer = None

        self.next_file()

    def __iter__(self):
        return self

    def next_file(self):
        if self.paths:
            warn('Processing ' + self.paths[0])
            self.handle = open(self.paths[0])
            del(self.paths[0])
        else:
            self.handle = None

    def next_line(self):
        if not self.handle:
            return None

        while True:
            line = self.handle.readline()
            if line:
                return line
            else:
                self.next_file()
                if not self.handle:
                    return None

    def next_record(self):
        if self.buffer:
            r = self.buffer
            self.buffer = None
            return r

        line = self.next_line()
        if not line:
            return None

        tokens = line.strip().split('\t')
        record = {}
        record['username'] = tokens[0]
        record['timestamp'] = tokens[1]
        record['offset'] = int(tokens[2])
        record['lifespan'] = int(tokens[3])
        record['cohort'] = int(tokens[4])
        record['namespace'] = tokens[5]
        record['revert_status'] = tokens[6]
        record['duration'] = float(tokens[7])
        record['bytes_added'] = int(tokens[8])
        record['bytes_removed'] = int(tokens[9])
        record['bytes_changed'] = int(tokens[10])

#        if CAP_YEAR:
#            if record['cohort'] + record['lifespan'] >= END_YEAR:
#                record['lifespan'] = (END_YEAR - 1) - record['cohort']

        return EditRecord(record)

    def unread_record(self, record):
        assert(self.buffer == None)
        self.buffer = record

    def next(self):
        userdata = []
        while True:
            record = self.next_record()
            if not record:
                if userdata:
                    break
                else:
                    raise StopIteration

            if userdata and userdata[0].username != record.username:
                self.unread_record(record)
                break

            userdata.append(record)

        is_current = not dt.before_fast(userdata[-1].timestamp, CURRENT_CUTOFF)
        return (is_current, userdata)

class WorkYearIterator(UserIterator):
    """
        Decorator around the user iterator.
        Format of next result is: is_current, user_data, by_year
        
        user_data is the result returned by the UserIterator
        by_year is a nested dictionary:
            year => dictionary with three types of keys:
                'edits', 'hours', 'bytes': dicts with keys namespaces, values metric values

                'edits_total', 'hours_total', 'bytes_total': sums of values for each metric

                'edits_prop', 'hours_prop', 'bytes_prop': dicts parallel to first, but with
                            proportions for each metric
        
    """
    def next(self):
        is_current, udata = UserIterator.next(self)
        by_year = {}

        for (year, year_records) in itertools.groupby(udata, lambda r: r['year']):
            year_metrics = {}
            for m in METRICS:
                year_metrics[m] = collections.defaultdict(int)
                year_metrics[m + '_prop'] = collections.defaultdict(float)
            for record in year_records:
                w = record.get_work_type()
                for m in METRICS:
                    year_metrics[m][w] += record.get_metric(m)
            for metric in METRICS:
                total =  sum(year_metrics[metric].values())
                year_metrics[metric + '_total'] = total
                year_metrics[metric + '_prop'] = {}
                for (k, v) in year_metrics[metric].items():
                    if total:
                        year_metrics[metric + '_prop'][k] = 1.0 * v / total
                    else:
                        year_metrics[metric + '_prop'][k] = 0.0
            by_year[year] = year_metrics

        return (is_current, udata, by_year)

if __name__ == '__main__':
#    for (is_current, ud) in UserIterator('dat/paper/active_test/'):
#        print 'user %s (current=%s) has %d edits' % (ud[0]['username'], is_current, len(ud))
    for (is_current, ud) in UserIterator('dat/paper/active_test/'):
        assert(ud[0]['cohort'] == ud[-1]['cohort'])
        assert(ud[0]['lifespan'] == ud[-1]['lifespan'])
        r = ud[0]
        if r['cohort'] == 2007 and is_current:
            print r['username'], r['cohort'], r['lifespan'], is_current
#        #print 'cohort %s year %s offset=%s, current=%s' % (r['cohort'], r['year'], r['offset'], is_current)
#    for (is_current, ud) in UserIterator('dat/paper/active_test/'):
#        print 'user %s (current=%s) has %d edits' % (ud[0]['username'], is_current, len(ud))
#    for (is_current, ud, by_year) in WorkYearIterator('dat/paper/active_test/'):
#        print 'user %s (current=%s) has %d edits' % (ud[0]['username'], is_current, len(ud))
#        print 'user %s (current=%s) has edits %s, prop %s' % (ud[0]['username'], is_current, ud, by_year)
#    for (is_current, ud, by_year) in WorkYearIterator('dat/paper/active_test/'):
#        #print 'user %s (current=%s) has %d edits' % (ud[0]['username'], is_current, len(ud))
#        if by_year.values() and by_year.values()[0]:
#            print by_year.values()[0]
#            r = by_year.values()[0][0]
#            if r['cohort'] == 2003:
#                print r['cohort'], r['lifespan']
